# Content type detection HTML, RDF-Dump, XML-Sitemap
# Expecting a URI, delivering "page", "sitemap", "dump" as results

"""
    Copyright (C) 2010 Uwe Stoll.
    Permission is granted to copy, distribute and/or modify this document
    under the terms of the GNU Free Documentation License, Version 1.3
    or any later version published by the Free Software Foundation;
    with no Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts.
    A copy of the license is included in the section entitled "GNU
    Free Documentation License".
""" 

import urllib2, re, xmlrpclib
from google.appengine.api.urlfetch import DownloadError



def get_uri_response(uri):
    if 'http://' in uri:
        req = urllib2.Request(uri)
        req.add_header('Accept-encoding', 'gzip')   
        try:
            response = urllib2.urlopen(req)
        except:
            return 'The URI could\'nt be resolved'
        else:
            uri = response.geturl()
            if ".gz" in uri:
                import StringIO, gzip
                compresseddata = response.read() 
                compressedstream = StringIO.StringIO(compresseddata)   
                gzipper = gzip.GzipFile(fileobj=compressedstream)      
                page = gzipper.read(1000)
            else:
                page = response.read(1000)
            return uri, page
    else:
        return 'Error: Please provide protocol (http://)'


def content_check(uri):
    #returns page, dump or sitemap / or error description
    resp = get_uri_response(uri)
    if type(resp) is str:
        return resp   
    else:
        uri, page = resp
    if 'htm' in uri:  
        return "page"
    elif 'xml' in uri:
        pass
    elif 'rdf' in uri:
        return "dump"
    elif 'owl' in uri:
        return "dump"
    else:
        pass
    if ('<html' or 'HTML') in page:
        return "page"
    elif ('<urlset' or '<URLSET') in page:
        return "sitemap"
    elif re.search('<\w*:RDF', page):
        return "dump"
    else:
        return "unclear"


def url_w3c_validate(uri):
    return urllib.urlopen('http://validator.w3.org/' + 'check?uri=' 
        + uri).info().getheader('x-w3c-validator-status') == 'Valid'

    
def pingxmlrpc(uri, service, submituri):
    server = xmlrpclib.ServerProxy(submituri)
    result = server.weblogUpdates.ping('',str(uri))
    if not result['flerror']:
        return service + ": Submission of " + str(uri) + " successfull."
    else: 
        return service + ": Submission of " + str(uri) + " failed.\012"+result['message']

def to_dict(model):
    import datetime
    import time
    SIMPLE_TYPES = (int, long, float, bool, dict, basestring, list)
    # Ack..
    output = {}
    for key, prop in model.properties().iteritems():
        value = getattr(model, key)
        if value is None or isinstance(value, SIMPLE_TYPES):
            output[key] = value
        elif isinstance(value, datetime.date):
            # Convert date/datetime to ms-since-epoch ("new Date()").
            ms = time.mktime(value.utctimetuple()) * 1000
            ms += getattr(value, 'microseconds', 0) / 1000
            output[key] = int(ms)
        elif isinstance(value, db.Model):
            output[key] = to_dict(value)
        else:
            raise ValueError('cannot encode ' + repr(prop))
    return output

"""
# TESTING STUFF
urilist = ("http://semantic.eurobau.com/sitemap.xml", 
           "http://semantic.eurobau.com/at/weber-terranova_at/company.rdf", 
           "http://www.la-mousson.de/", 
           "http://www.la-mousson.de/sitemap.xml",
           "http://www.semantium.de/research/rdftest.xml",
           "http://www.jarltech.de/semweb.rdf",
           "http://www.bloldldofl.o",
           "http://www.semantium.de/wrong.ph2")

for uri in urilist:
    print uri,":", content_check(uri)
"""